{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rental Listing Inquiries\n",
    "根据公寓的特征来预测其受欢迎程度（用户感兴趣程度分为高、中、低三类）。其\n",
    "中房屋的特征 x 共有 14 维，响应值 y 为用户对该公寓的感兴趣程度。评价标准\n",
    "为 logloss。数据链接：https://www.kaggle.com/c/two-sigma-connect-rental-listing-inquiries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入必要的工具包¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Columns: 228 entries, bathrooms to interest_level\n",
      "dtypes: float64(9), int64(219)\n",
      "memory usage: 85.8 MB\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 在此案例中，为了提高运行速度，选择稀疏格式文件\n",
    "dtrain = pd.read_csv('./data/RentListingInquries_FE_train.csv')\n",
    "\n",
    "# 查看数据情况\n",
    "print(dtrain.info())\n",
    "dtrain.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = dtrain['interest_level']\n",
    "X_train = dtrain.drop('interest_level', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/fanhong/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train_part, X_train_rest, y_train_part, y_train_rest = train_test_split(X_train, y_train, train_size = 0.33,random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 16286 entries, 30062 to 2732\n",
      "Columns: 227 entries, bathrooms to work\n",
      "dtypes: float64(9), int64(218)\n",
      "memory usage: 28.3 MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(X_train_part.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 初步确定弱学习器数目和学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 采用缺省的默认学习率（learning_rate =0.1）并由此计算出对应的最优学习器数目\n",
    "# 由于评价标准为logloss，目标函数采用multi:softprob\n",
    "# 将模型适配的过程封装为一个函数，方便调用\n",
    "def modelfit(alg, X_train, y_train, cv_folds = None, early_stopping_rounds=10, filename='nestimators_temp.csv',predict=False):\n",
    "    # 设置分类类别数\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    print('n_estimarors:',xgb_param['n_estimators'])\n",
    "    # XGBoost直接引用函数“cv”可以在每一次迭代中使用交叉验证，并返回理想的树数量\n",
    "    xgbtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "    cvresult = xgb.cv(xgb_param,\n",
    "                      xgbtrain,\n",
    "                      num_boost_round = alg.get_params()['n_estimators'],\n",
    "                      folds=cv_folds,\n",
    "                      metrics='mlogloss',\n",
    "                      early_stopping_rounds=early_stopping_rounds)\n",
    "    #将结果阶段性保存，避免重复工作，并将数据以csv文件的形式传递\n",
    "    cvresult.to_csv(filename, index_label = 'n_estimators')\n",
    "    #得到交叉验证优化后的树数量\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    print('优化后n_estimators:',n_estimators)\n",
    "    #根据该数量优化学习率\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "    \n",
    "    if predict:\n",
    "        train_predprob = alg.predict_proba(X_train)\n",
    "        logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        print('logloss of train is:', logloss)\n",
    "    #将cvresult返回，便于后续操作\n",
    "    return cvresult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgb1.get_params() {'base_score': 0.5, 'booster': 'gbtree', 'colsample_bylevel': 0.7, 'colsample_bytree': 0.8, 'gamma': 0, 'learning_rate': 0.1, 'max_delta_step': 0, 'max_depth': 6, 'min_child_weight': 1, 'missing': None, 'n_estimators': 1000, 'n_jobs': 1, 'nthread': -1, 'objective': 'multi:softprob', 'random_state': 0, 'reg_alpha': 0, 'reg_lambda': 1, 'scale_pos_weight': 1, 'seed': 3, 'silent': True, 'subsample': 0.5, 'eval_metric': 'mlogloss'}\n",
      "xgb1.get_xgb_params: {'base_score': 0.5, 'booster': 'gbtree', 'colsample_bylevel': 0.7, 'colsample_bytree': 0.8, 'gamma': 0, 'learning_rate': 0.1, 'max_delta_step': 0, 'max_depth': 6, 'min_child_weight': 1, 'missing': None, 'n_estimators': 1000, 'objective': 'multi:softprob', 'reg_alpha': 0, 'reg_lambda': 1, 'scale_pos_weight': 1, 'seed': 3, 'silent': 1, 'subsample': 0.5, 'eval_metric': 'mlogloss'}\n",
      "n_estimarors: 1000\n",
      "优化后n_estimators: 125\n"
     ]
    }
   ],
   "source": [
    "# 初始化XGBClassifier\n",
    "xgb1 =XGBClassifier(\n",
    "    max_depth=6,\n",
    "    learning_rate =0.1,\n",
    "    n_estimators=1000,\n",
    "    objective='multi:softprob',\n",
    "    eval_metric='mlogloss',\n",
    "    booster='gbtree',\n",
    "    min_child_weight=1,\n",
    "    nthread=-1,\n",
    "    gamma=0,\n",
    "    seed=3,\n",
    "    subsample=0.5,\n",
    "    colsample_bytree=0.8,\n",
    "    colsample_bylevel=0.7)\n",
    "print('xgb1.get_params()',xgb1.get_params())\n",
    "print('xgb1.get_xgb_params:',xgb1.get_xgb_params())\n",
    "\n",
    "# 利用StratifiedKFold配置交叉校验\n",
    "# StratifiedKFold用法类似Kfold，但是他是分层采样，确保训练集，测试集中各类别样本的比例与原始数据集中相同\n",
    "# n_splits：表示划分几等份；shuffle：在每次划分时，是否进行洗牌；random_state：随机种子数\n",
    "kfold1 = StratifiedKFold(n_splits=5, shuffle=True)\n",
    "\n",
    "cvresult1 = modelfit(xgb1, X_train_part, y_train_part, cv_folds= kfold1, filename='1_nestimators.csv')\n",
    "n_estimators = cvresult1.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def showlogloss(cvresult=None,fromfile=True,datafile='nestimators_temp.csv', figfilename='n_estimators_temp.png'):\n",
    "    if fromfile:\n",
    "        cvresult = pd.DataFrame.from_csv(datafile)\n",
    "    \n",
    "    # plot\n",
    "    test_means = cvresult['test-mlogloss-mean']\n",
    "    test_stds = cvresult['test-mlogloss-std'] \n",
    "\n",
    "    train_means = cvresult['train-mlogloss-mean']\n",
    "    train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "    x_axis = range(0, cvresult.shape[0])\n",
    "\n",
    "    plt.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "    plt.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "    plt.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "    plt.xlabel( 'n_estimators' )\n",
    "    plt.ylabel( 'Log Loss' )\n",
    "    plt.savefig( figfilename )\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XecFPX9+PHXe3evcRwc5Wh3NOkgCIhgBxVjSWIvEDUakxiNxvK1RJP8UkxM01iixhKjJiaKikaJGhvWWDmkKCC93NHujnZ3wNV9//74zC3LeQ24vdm9fT8fzsOdmc/OvGeW2/fO5zPz+YiqYowxxgAE/A7AGGNM/LCkYIwxJsKSgjHGmAhLCsYYYyIsKRhjjImwpGCMMSbCkoIxUUTkJyLyiN9xGOMXSwoJRkQ6isgaEflW1LIsEVknIudELZsgIi+JyDYR2S4ii0XkNhHp4q2/RERqRaTcm1aJyBUxjn2KiBTGch/7oqF4VPW3qvq9GO1vjYhMjcW2Y6GtPq9EOy/tnSWFBKOq5cBlwD0ikuMt/iOQr6ozAUTkSOAd4ANguKpmAycDNcAhUZv7SFU7qmpH4BzgjyIyrm2OxOwLEQn5HYNJEqpqUwJOwOPAU8AUYAvQO2rd/4B7m3n/JcD/6i37FPhW1PxpwCJgOy7JjIhaN8Jbtt0rc1rUulOBxUAZsB64AcgEdgNhoNyb+jRyXPcDL3vv/wQY1ILzMRx4A9gKLAXO2594gF8C//TeNwBQ4DtAAbANuBw4DFjoHft9UfsZBLzlfR4lwL+AbG/dE96+dnv7uqkF53gN8GNvX5VAyJtf7x3LUuCEBs7F4cAmIBi17Exgofd6IpAPlAKbgTsbOadTgMJG1nUG/gEUA2uBnwEBb10Q+JN3DlYDV3nnMdTIttYAUxtZ931ghfe5zqr7NwMIcBdQBOzwztHBjX3efv+9JtLkewA27ecHB12Ajd4f3neilmcCtcCUZt5/CVFJwfui2w4M9eaHAjuBE4EU4CbvjzPVm18B/MSbP977AxzmvXcjcExUnOO9141+yUTF8bj3BTDR+xL8FzCjmfdk4r60v+O9Z7x3Xkbtazw0nBQeBNKBrwEVwAtADyDX+1Ka7JUf7J2vNCAHeA+4O2rbe335NXWOo8rPB/oCGcAw7zj7RMXXYMIEVgInRs0/C9zsvf4IuMh73RE4vJFtNPp54RLCi0CWF8cy4LveustxX8p53vl+k/1ICt6/qxLv80wD7gXe89adBMwFsnEJYgTeD6PGPm+bWjZZ9VGCUtVtuF+YHYDno1Z1wVULbqpbICJ/9NoVdorIz6LKHu4tL8ddJTwBLPfWnQ+8rKpvqGo1cAfui+lI3C/RjsDvVbVKVd8CXgKme++tBkaKSCdV3aaqn+3j4T2vqp+qag0uKYxtpvw3gDWq+piq1nj7ew5XJdYa8fxaVStU9XXcl/hTqlqkquuB94FxAKq6wjtflapaDNwJTG5iu02d4zp/VtUCVd2NS/Zp3rGkqOoaVV3ZyLafwvs8RCQL9+v5qajzMVhEuqtquap+vC8nQ0SCXuy3qGqZqq7BXRlc5BU5D7hHVQu9f6e/35ftR7kAeFRVP1PVSuAW4AgRGeAdQxbuClFUdYmqbow6vgP5vJOaJYUEJSIX4n6hvQn8IWrVNlw1Re+6Bap6k7p2hX/jfknX+VhVs9W1KfQCRgG/9db1wVUL1G0jjPuVmuutK/CW1VnrrQM4G/cltFZE3hWRI/bx8DZFvd6FS0BN6Q9M8hLcdhHZjvtC6dVK8WyOer27gfmOACLSQ0RmiMh6ESkF/gl0b2K7TZ3jOgVR61cA1+KuZoq8ffVpZNtPAmeJSBpwFvCZqtbt67u4q5QvRWSOiHyjiRgb0h13hbg2aln0598nOu56r/dF/fNTjquay/V+iNyHq2rcLCIPi0gnr+iBft5JzZJCAhKRHrj61O8DPwDOE5FjAVR1J64e/qx92aaqbsb9uv6mt2gD7su2bp+Cq8ZY763rKyLR/376eetQ1TmqejquiuUF4Jm63exLTPugAHjXS3B1U0dVvaKN4/mdt80xqtoJuBBXtVGn/v6aOscNvkdVn1TVo733KXv/IIgutxj3hXoK8C1ckqhbt1xVp+POxx+AmSKS2fLDpAT3a7x/1LLI54+rvsmLWtd3H7Ydrf75yQS6seff2Z9V9VDcj5mhwI3e8sY+b9MClhQS033AC6r6tnfJfBPwV+9XId78pSJys5dAEJE8YGBjGxSRbrjGyEXeomeAr4vICSKSAlyPa+z8EJd0dgI3iUiKiEzBJZMZIpIqIheISGevSqQUV+0B7hd2NxHp3Ernoc5LwFARuciLJ0VEDhOREW0cTxauEXm7iOTifUlF2QwcFDXf1Dn+ChEZJiLHe59zBe4qpbahsp4ngauBY3FtCnXbuVBEcrwrk+3e4ka3IyLp0RPuSvQZ4Dbvduj+wP/hrozqjusaEckVkWxc43hzUurtJ+TF/x0RGesd82+BT1R1jff5TvLO207vfNQ283mblvC7UcOmfZuAM3C/oLLrLZ8N3BY1Pwl4BfdHvx34ArgN6OatvwT3x1J3500Rrs65R9Q2zsQ1GO4A3sVruPXWjfKW7fDKnOktTwVexVVjlQJzgKOj3vcorgpgO43fffSbqPkpNNM47ZUbhrtjqdjb/lu4toh9ioeGG5pDUeULiWrEx30R/izqnMz1zud83Jd8YVTZ04F13r5uaME5XsPeDdNjcG0/ZbjG+JcaOodR5fvhvsBfrrf8n97nXY77EXBGI++f4h1//Wkwru3qn975LgB+zp67j0K4K9ktuLuPrsNdWUgj+1nTwD5+4627HNdoXne8ed7yE3B3HJWz506vjs193jY1P4l3go0xJiZE5BTgQVXt32xh4zurPjLGtCoRyRCRU0Uk5FWj/QJ3k4NJAHalYBKCiBwD/LehderunjJxQkQ64KrChuPaPV4GrlHVUl8DMy1iScEYY0yEVR8ZY4yJSLhOtrp3764DBgzwOwxjjEkoc+fOLVHVnObKJVxSGDBgAPn5+X6HYYwxCUVE1jZfyqqPjDHGRLGkYIwxJsKSgjHGmAhLCsYYYyIsKRhjjImwpGCMMSbCkoIxxpiIpEkK5ZU1fLF+h99hGGNMXEuapPDtO5/n7vvvYWdljd+hGGNM3EqapPC7wAM8kvon1qzf2HxhY4xJUkmTFDpN/iEARau/8DkSY4yJX0mTFLoPGAXAzg1f+hyJMcbEr6RJCindB1FLALas8DsUY4yJW0mTFAilsSXUi47lq/2OxBhj4lbyJAWgvONAelYVUFUT9jsUY4yJS0mVFMLdBjNANrGmpMzvUIwxJi4lVVLI6D2MDKmicK21KxhjTEOSKil0738wADvWLfY5EmOMiU8xSwoi8qiIFIlIgw8GiPNnEVkhIgtFZHysYqmT1nMoADXFy2K9K2OMSUixvFJ4HDi5ifWnAEO86TLggRjG4mT1YrdkkLZjVcx3ZYwxiShmSUFV3wO2NlHkdOAf6nwMZItI71jFA4AI2zL603X3WmrDGtNdGWNMIvKzTSEXKIiaL/SWfYWIXCYi+SKSX1xcfEA7rc4+iAGykcJtuw5oO8YY0x75mRSkgWUN/nxX1YdVdYKqTsjJyTmgnab0HEYftrBqQ8kBbccYY9ojP5NCIdA3aj4P2BDrnWbnjSQgyt9fmh3rXRljTMLxMynMAr7t3YV0OLBDVWPer3WHPsMBmNzNBtwxxpj6QrHasIg8BUwBuotIIfALIAVAVR8EXgFOBVYAu4DvxCqWvXQbBECgcE6b7M4YYxJJzJKCqk5vZr0CV8Zq/41KzWR7Wh+679pCRXUt6SnBNg/BGGPiVVI90VynsutwhkoByzZbH0jGGBMtKZNCeu5oBspGviywO5CMMSZaUiaFrP6HEJIwxasX+h2KMcbElaRMCoGebmjO2k2LfI7EGGPiS1ImBboNokZSyNyxlLB1d2GMMRHJmRSCKZRlHcSg8DrWbrXuLowxpk5yJgVAeoxkWKCAxRtK/Q7FGGPiRtImhY79DqG3bGVlQUHzhY0xJkkkbVII9XajsO1cZ3cgGWNMnaRNCvQYCUCweInPgRhjTPxI3qTQqQ+VoSxyq1azccduv6Mxxpi4kLxJQYTqbiMYFihg/rrtfkdjjDFxIXmTApCxs5Dhso5565oaNdQYY5JHUieF4PE/oaNUsGm1PdlsjDGQ5EmBPuMASNm8gOrasM/BGGOM/5I7KeQMpzaQxghdyZcbrRttY4xJ7qQQDFHTczSjA6uZV7DN72iMMcZ3yZ0UgNS+4xkdWMOCtVv8DsUYY3yX9ElB+oynAxVsWWeNzcYYk/RJoa6xuev2RWzbWeVzMMYY4y9LCt2HUBvqYO0KxhiDJQUIBKH3IRwSWMWcNZYUjDHJLaZJQUROFpGlIrJCRG5uYH1/EZktIgtF5B0RyYtlPI0J5o5nVGAtn64s8mP3xhgTN2KWFEQkCNwPnAKMBKaLyMh6xe4A/qGqY4Bbgd/FKp4m9RlHGlXs3rCInZU1voRgjDHxIJZXChOBFaq6SlWrgBnA6fXKjARme6/fbmB92/Aamw9mJXPWWD9IxpjkFcukkAtED2tW6C2LtgA423t9JpAlIt3qb0hELhORfBHJLy4ubv1Iuw1CM7pyWHA5H6+ypGCMSV6xTArSwDKtN38DMFlE5gGTgfXAV+pvVPVhVZ2gqhNycnJiEKkgfSdxZMpyPl5lD7EZY5JXLJNCIdA3aj4P2BBdQFU3qOpZqjoO+Km3bEcMY2pcv8PJrV3PhvXrKLd2BWNMkoplUpgDDBGRgSKSCkwDZkUXEJHuIlIXwy3AozGMp2n9jgBgLEutXcEYk7RilhRUtQa4CngNWAI8o6qLRORWETnNKzYFWCoiy4CewG2xiqdZfcaiwTQmBpdZFZIxJmmFYrlxVX0FeKXesp9HvZ4JzIxlDC0WSkNyx3PY2mV866O13HLKCL8jMsaYNmdPNEfrdzgHB1YRrt7FlvJKv6Mxxpg2Z0khWr8jCGoth8hK3l9e4nc0xhjT5iwpRMs7DIBj0lby9lLr8sIYk3wsKUTr0BVSOnA8n/LusmJqw/UfqzDGmPbNkkJ9aZ0Yomsp37Wb+QXb/Y7GGGPalCWF+k79IyFqGBdYyTtWhWSMSTKWFOobcAwgnNPV2hWMMcnHkkJ9HbpCn7EcE/iCL9aXUlRW4XdExhjTZiwpNOSg4+hd9gWZ7Ob8Bz/yOxpjjGkzlhQactAURGv4ZudV9O2W6Xc0xhjTZiwpNKTvJAilc3aXFXy4ooTtu6r8jsgYY9qEJYWGpKRDvyMYXTmPmrDy+uLNfkdkjDFtwpJCYw6aQvq2ZRySvZv/fr7R72iMMaZNWFJozKDjAPhe7Uz+t6KEHburfQ7IGGNiz5JCY3qNgazeHJ0rVNcqs5dYFZIxpv2zpNAYERjyNbI3vE9mMMyt/1nsd0TGGBNzlhSaMuwUpKqMnx28lbLKGorLbIwFY0z7ZkmhKQMnQyidU1LnUxtWXpi33u+IjDEmpiwpNCW1AwycTHbBbMbmdebZuQWoWnfaxpj2y5JCc4aeBNvX8t3hVSzbXM7Cwh1+R2SMMTFjSaE5Q08G4MTQPNJCAZ6dW+BzQMYYEzuWFJrTORdSMkn/6C5OPrgXs+ZvoKK61u+ojDEmJmKaFETkZBFZKiIrROTmBtb3E5G3RWSeiCwUkVNjGc9+O/o6qCxjy4ZVlFbU8Io94WyMaadilhREJAjcD5wCjASmi8jIesV+BjyjquOAacBfYhXPATn4LACeOHwjg3IyefzDNdbgbIxpl2J5pTARWKGqq1S1CpgBnF6vjAKdvNedgQ0xjGf/dRsEvcYgi/7NxUcOYGHhDubZ+M3GmHYolkkhF4hulS30lkX7JXChiBQCrwA/amhDInKZiOSLSH5xcXEsYm3eqDNhfT5nHxSmY1qIv3+4xp84jDEmhmKZFKSBZfXrXKYDj6tqHnAq8ISIfCUmVX1YVSeo6oScnJwYhNoCo84EIHPFfzjn0Dxe+XyjDdVpjGl3YpkUCoG+UfN5fLV66LvAMwCq+hGQDnSPYUz7r+tA6DMevniei48cQHWtcub9H/gdlTHGtKpYJoU5wBARGSgiqbiG5Fn1yqwDTgAQkRG4pOBT/VAL7N4KG+czUDaRnZHC5tJKdlbW+B2VMca0mmaTgogMEpE07/UUEblaRLKbe5+q1gBXAa8BS3B3GS0SkVtF5DSv2PXA90VkAfAUcInG8209HbyqqwUz+Nslh1ETVp78ZJ2/MRljTCuS5r6DRWQ+MAEYgPuCnwUMU1VfnimYMGGC5ufn+7Fr54kzoWQ5XLOQCx79lGWby3n/puNITwn6F5MxxjRDROaq6oTmyrWk+ijs/eo/E7hbVa8Deh9ogAlr7AWwowDWvMdVxw2huKySZ/Kt6wtjTPvQkqRQLSLTgYuBl7xlKbELKc4N/zqkdYb5T3L4QV3pmBbiNy8toaom7HdkxhhzwFqSFL4DHAHcpqqrRWQg8M/YhhXHUjJg9NmweBZSWUpudgZVtWGe+Hit35EZY8wBazYpqOpiVb1aVZ8SkS5Alqr+vg1ii19jL4Sa3fDw8bx23bEcOzSHP89ezvZdVX5HZowxB6Qldx+9IyKdRKQrsAB4TETujH1ocSx3PKR0gPJNAPz01BGUVVRzz+zlPgdmjDEHpiXVR51VtRQ4C3hMVQ8FpsY2rDgnAlN/CVXlsP4zhvXKYtrEfjzx0VpWFZf7HZ0xxuy3liSFkIj0Bs5jT0OzOWQapGTCnEcAuG7qUMKqnPmXD60HVWNMwmpJUrgV93zCSlWdIyIHAVZPkt4ZDjkfvngOdm0lJyuNn359JDt2V/PqF5v8js4YY/ZLSxqan1XVMap6hTe/SlXPjn1oCeCw70FNBcx7AoCLj+jPyN6d+OV/FlFu3V8YYxJQSxqa80Tk3yJSJCKbReQ5Eclri+DiXs9RkNYJ3r4NwrWEggFuO/NgisoqufP1ZX5HZ4wx+6wl1UeP4bq26IMbD+E/3jIDkNUHairhS9fcMq5fF3I6pvHoB6uZu3arz8EZY8y+aUlSyFHVx1S1xpseB3wa1CAO/fBD6DoI3rsDvAbm2ddPJjc7g+ueXmC9qBpjEkpLkkKJiFwoIkFvuhDYEuvAEkYgCEdfB5sWwoo3AchKT+Gu88dSsG0Xv35psc8BGmNMy7UkKVyKux11E7AROAfX9YWpM+Z86NwX3rs9crUwcWBXfnDsIGbMKeDEO9/1OUBjjGmZltx9tE5VT1PVHFXtoapn4B5kM3VCqXDUNVDwCaz5X2Tx/504lMzUIKtKdrKmZKePARpjTMvs78hr/9eqUbQH4y6EYArM+FbkaiE1FODVa48lKz3E5f+cy+6qWp+DNMaYpu1vUpBWjaI9SMmAr90GlaWwek91Ud+uHbhn2ji+3FTG0X94y552NsbEtf1NCvbN1pBDL4FOeTD715GrBYDJQ3PI65LBlp1V3PvWCv/iM8aYZjSaFESkTERKG5jKcM8smPpCaTD5RlifD8te22vV+zcdx9nj87jzjWXMnFvoU4DGGNO0RpOCqmapaqcGpixVDbVlkAll7AUQSoeZl0J4z2hsIsLvzhrNUYO7ceOzCzjpLrsjyRgTf/a3+sg0JpgCnftB9U5YOGOvVamhAA9ceCgZqUGWFZXz3rJin4I0xpiGWVKIhSs/gdwJ8OYvobJsr1Wd0lP44MfHM7xXJy5+7FNOvvs9f2I0xpgGxDQpiMjJIrJURFaIyM0NrL9LROZ70zIR2R7LeNpMIACn/AHKN8P7Xx2krktmKv/63iQyUoIs3VzG64usq21jTHyIWduAiASB+4ETgUJgjojMUtVIvw+qel1U+R8B42IVT5vLmwBjpsFH98H4b0PXgXut7pqZyvBeWSzdXMYV//qM/l07kJOVxtM/OMKngI0xpmVdZzd0F1KB1532QU28dSKwwht/oQqYAZzeRPnpwFP7Fn6cm/oLCNfAw8ftdYtqned/eBSf/mQqRw7qxqqSnazfvtueYzDG+Kol1Ud3Ajfius3OA24A/or7kn+0ifflAgVR84Xesq8Qkf7AQOCtRtZfJiL5IpJfXJxAjbOd+rgH2iq2uRHaGpCZFuKRiyfQLTOVwm27Oey2Nzn3wQ/bOFBjjHFakhROVtWHVLVMVUtV9WHgVFV9GujSxPsaeuq5sZ/B04CZqtpgPxCq+rCqTlDVCTk5CdZr96QfQJ9x8OrNsKvh8RXSQkHyfzaV66YOpaS8iiUbyyjYuquNAzXGmJYlhbCInCciAW86L2pdU3UdhUDfqPk8YEMjZafR3qqO6gSCcNq9sLMY7p/UaDER4ZqpQxick8muqhqm3PEOU//0TtvFaYwxtCwpXABcBBR500XAhSKSAVzVxPvmAENEZKCIpOK++GfVLyQiw3BXHB/tY+yJo9do1/3FziJY+mqTRd+8fgrv3HAc6SkBVhTv5MonP6OorKKNAjXGJLuWdJ29SlW/qardvembqrpCVXer6v+aeF8NLmm8BiwBnlHVRSJyq4icFlV0OjBD23sL69WfQc/R8OKVUF7UZNF+3Towsncn8rIzeGPRZo743Vscd/vbhMPt+xQZY/wnzX0Xi0gecC9wFK666H/ANarqSwc+EyZM0Pz8fD92feCKlsBDk2HQcTB9Bkjznc2uLC7njPs/oKyihszUIAO6Z/Ly1ce0QbDGmPZEROaq6oTmyrWk+ugxXLVPH9zdQ//xlpl91WMEnHgrLHsV/jy+RW8ZlNOREb2yGJSTSVVtmEUbSrnh2QVsLrUqJWNM62vJlcJ8VR3b3LK2ktBXCuA6ybv9INi9HS59Dfo13vhc39kPfMiG7bvZUl5FTThM787pvPSjY+iSmRrDgI0x7UFLrxRakhTeBB5nz91B04HvqOoJBxrk/kj4pAAuITw8BWoq4LJ3IavnPr197ZadnHn/B2zdVU1AoEdWOs9efgR9u3aITbzGmITXmtVHlwLnAZuAjcA5wHcOLLwkl5EN5z/h+ka6bwLUVO3T2/t3y2RIzyxG53aiS4dUNpVWcOztbzP21tc56a53rUHaGLPfmr1SaPBNIteq6t0xiKdZ7eJKoc59h0HJMje+82n3tajhub7zH/qIypowxwzpzgPvrKQmrKSFAuRkpfHcFUdy9VPzAKxPJWOSXKtVHzWy8XWq2m+/IjtA7SopALz1G3jvdvjab+DIHx3Qps598EO27ayiqKyS0ooaAuK66u6elUaXjBQCAbHkYEySamlS2N9eUvf9J61p2JSfuKuF138G8/4FV36835t69vIjAXf1UFFdy1GDu/PX91exvaiaoAhdMlOYvWQzD76z0hKEMaZBdqUQD6p2wR1D3Whtl7wC/Vvvy/q8Bz+ktKKGkvJKtu2qpjasBAQ6Z6SQ3SGFzhmpvHDlUa22P2NMfDrg6iMRKaPhvo0EyPBrnOZ2mRQAdm6BR7/m+ki69HXoMbxVN3/+Qx8RVuXK4wZz08yFbN9VTVWtG0M6PSVA54wUbj39YB5+dyWhYMCuIoxpZ2LapuCndpsUALatcXcjIfDDj6HboJjs5vyHPkJV2V1dy47d1ezYXUNZRTV1Ny11SA3SKT2FThkhnvz+4Vz5r88Aa6w2JpFZUkhUmxfB378JwTS45KWYJYZodVcRN508nOtmzKe0opqyyprIuECpoQCZqUEyUoN0SAnyyMWHcdPMBYhYu4QxicKSQiLb9AU8PBkkAFd8BN0Ht9muz3/IdVYbDivllTWcOT6XB99dxa6qGiqqw5FyQRE61CWK1CB3nHsIv/vvEkIBq3oyJh5ZUkh0D012Vw3pneHCmW6gHp9EJ4rd1bVcdER//vT6MnZX1bKrqpbaqH9DqcFAJFlcfcIQ/v7hGtJDAVJDASTqOQxLHMa0LUsK7UHJCnjiTChdDzkj4Icf+B1RRF2iUFWqasL86IQh/GLWokiiqKiu/cpdCqlBlxxSQwEuPLw/ryzcQHpKkLSUQOQKo267ljSMaV2WFNqL0o1w73io3u1GcBt/kd8RNauuIfvO88fyvb/nU1lTS2V1mKraMFU1YSprwlTXhonujSMYEAZ060BRaSWpoQBp3nTzqSO4581lpAYDpIQCBKwdw5j9YkmhPanYAc9eAivfciO4Xfs5BFrSbVX8qbsS+PulEzn3wQ+pqA5TUV1LZU2YCQO68P7ykkjiaEhKUBjRuxPrtu4iJSCEggFCAeGq4wfzyPurCQaEkDcFvemZy4+M7LeOJRaTbCwptDe1NXDXKCjfBMO/AWc+BGkd/Y6q1e1VLVWr/OaMg7n+mflU1bpEUVUTZlSfzsxZs5Xq2jA1tdrkQOHgnsOoqdVIkggFhGOG5DBnzdY9SSQo3HLKCO5+cxkpwQApwQBB76nvhhKKVXOZRGNJoT1ShU8egtdugVA69BgJ35/td1S+iE4eYYV7po/jB//Ipzas1IQ16v9hvj6mDy/MW09t1PJendNZU7KT2nDjSSUgkJOVxo7d1YQCgcjVx9QRPXlnWREBEYIiBAIgIgREuHbqEO5/awWBgJsPBuDBiyZw9VOfERSJNLY3l2waWmYJyBwISwrt2Yo34V/nuWfLz3kcRp7W3DuMJ/oLtq7tI6xQEw7z69NHc8Oz86muVaprw1TXKpOH5vD64k17JZTsDimUlFcRbiKhNCYYcImkd3Y6m0srEAQR10HumNxslmwsRQQC4pafOLInby0p2ivJBLwEdM3UIdw7e3mkd10BfnzycG5/7UsvSbmlvz7jYP7fC58jCN5/3D1tLNc/s8BtK+DKPn3ZEUz/a8N9b+1PEtvXZa21XdMwSwrt3dZVMPNS2DAPOvaCqz+D1Ey/o2qXmvoyCqsSVkXVvb7j3LFcO2MetWG3vFbh+8cM5P63V+x1FXP04O68t7wEVZdYVJWhPbNYtKE0si1V6Nwhha07qyLzbc3LK3RICVJRE470hCle/1nbd1VH5gG6Zqaydefe44Nkd0hl284qLwGBIOR2yaBw6656O3LrhvbsyKrinXv1JD86rzOfF+7Ya7tj8rJZWLh9zyZEGN8vm3kF2/fECRw9JIcPV5Yp8CHTAAAWk0lEQVREEvCe8nDGuFxeWrDRS6Bu2QWT+vPPj9e6hK+ur5+zx+cyc26hl8AFAS483JUTIbLty44dxF/fX4VEnRPxNnzD14bxp9eXRvavwI0nuWXixQ/wk1NHcNvLiyNHIAI//8ZIfv3SYtJTgvx7P/sqs6SQDGqq4J5DoGwDdD0IzngA+h3ud1SmCfWvVKI1t0y9JBP2Es5d54/lmhnzImVV4bdnjebm5xai3peZqnLLqSP43StLXBmv3NUnDOGuN5ZFklpY4dxD+/J0/ro9O9c9nZ99fXRv/rNww17Lpo7syZuLN++1/xNG9GD2kqK9jmHqyJ7MXrI5sh9V5chB3fl41ZboXYGXIA/pm83ctdv22tfwXll8ualsr+0O65XF0qhlijIopyMrisoj8ShKr07pbNheQd11Xd1XngKhgDR6U0M8GtCtA+/ceNx+vdeSQjJZ/R68eCVsXwdZfdxVQ0qG31GZBNFQoqqrhjmQJLavy2K13eb2dd6DH0aSKLg74y5+9NPIVZJA5A62PVd28I/vTuTbf/t0r4T2wIWH8oN/5Ee2tWe7yh/OPoSbZi4A9lxF/O6sMfz4uYXUfQ+rwq/POJifv7iobgso8MtvjuKXsxYl/pWCiJwM3AMEgUdU9fcNlDkP+CXu3C1Q1W81tU1LCo2oLIN7J7i7k7oNhtP/Av0m+R2VMSZOtOYYzfsbQBC4HzgFGAlMF5GR9coMAW4BjlLVUcC1sYqn3UvLghuWwrdfhO0Frhvul2+AilK/IzPGJJBYPgE1EVihqqtUtQqYAZxer8z3gftVdRuAqhZhDsxBU+CmlTDpcpjzCNw+CO6biC+tlMaYhBPLpJALFETNF3rLog0FhorIByLysVfd9BUicpmI5ItIfnFxcYzCbUfSsuCUP8D3ZkMgBUqWwqMnQeFcvyMzxsS5WCaFhsZxrv9zNQQMAaYA04FHRCT7K29SfVhVJ6jqhJycnFYPtN3KOxRuKXB9Jq3/DB45Hl74IZRtbv69xpikFMukUAj0jZrPAzY0UOZFVa1W1dXAUlySMK0lEITx34Yfr4ajroXPn4U7R8DdY6C6wu/ojDFxJpZJYQ4wREQGikgqMA2YVa/MC8BxACLSHVedtCqGMSWvtCw48VdumM/0zrB9Lfy+H9x7GIRr/Y7OGBMnYpYUVLUGuAp4DVgCPKOqi0TkVhGp65fhNWCLiCwG3gZuVNUtDW/RtIpug9xVw8X/gWAItiyDvxwOn8+EcOI8xGOMiQ17eC2ZhcOwZBa8cAVU73Id7B33E9cLqzTUJGSMSVS+P6dgEkAgAKPOgFvWw9l/g9pqePpC+G2uu3KorfE7QmNMG7OkYFxyGH2Oa2/oNgQIw3Pfhd/2gQ/+DLu2+h2hMaaNWPWR+apwGB440o0NXVkKEoAOOXDB09BnnN/RGWP2Q0urj0JtEYxJMIEAXOn1q795ETxxFuwsgoenQGoWnPEX1+6QoEOCGmMaZ0nBNK3nKNenUsUOeGiK66b7mYsglAGd+sAP3nW3uxpj2gX7qWdaJr0zXDMPfrIBznnMXSVsXQm/7w93DIdNX/gdoTGmFdiVgtk3gSAcfBaMOhPWz4Wnprvuuh88yl0xZPWBKz6AYIrfkRpj9oNdKZj9IwJ5E+DG5XDTKjjpt+6W1pKl7q6luw+BnfYcojGJxu4+Mq0nHIblr8Pzl0HlDgimQXonyMyBy96FUKrfERqTtOzhNdP2AgEYdjLcss498zD+2+4Zh6LFcFsv+ONg+OI512htjIlLdqVgYqu6Ala9A7N+BLu3QrgGEBj+dRh7AQw50dofjGkDcTFGcyxYUkhg4Vp3W+uuEghXw85iCIRg9Hkw8jQYOBlSO/gdpTHtkj28ZuJPIAhXvO9e11bDg8e6BPH507DgSdcGkdIBOnSFS1+DjjagkjFtzZKC8UcwBa78yL2uqYK1/4Plb0D+o+75hzsGQ1pnmPpzGHEadOzhb7zGJAmrPjLxRdV1rfHUNNhZAjW73fL0bDjljzDiG5Ca6W+MxiQga1MwiU/V3bn05PlQXgS1la5zvsEnwtCvwZCvQXY/v6M0JiFYm4JJfCKu76XrvnDPQDzktUEUfwnLX3NlUjpARhc462Hoe7gbTc4Ys9/sL8gkhkAArvife60KJctcG8R7t0PpBnj86+5OpoyucPp9cNAUCKX5GbExCcmqj0ziqyyDlW/Dy9fDri2gtSBB14lfRheY9iR0H2pdfZukZm0KJjnVVLpnIXZvhd3bXDsEuKuItE4w+ccw8FjIGW5JwiQVa1MwySmUtudWV1XYugrWfQRv3ur6Y3r1x25dRlfXZpHWCc75G/QaY09WG4NdKZhks30drH4f1n4An8/ccyVRd1fToONg8FToNtglDWPaibioPhKRk4F7gCDwiKr+vt76S4DbgfXeovtU9ZGmtmlJwbSq0g3wjzNcJ32pme7BOYBQOqR3gVP/CAOOdk9ZG5PAfK8+EpEgcD9wIlAIzBGRWaq6uF7Rp1X1qljFYUyTOvWBqz7dM79tLax4E976Dezc7IYeBXfra3pn95T1pf+1JGHarVi2KUwEVqjqKgARmQGcDtRPCsbEjy794bDvuqm2GtZ/Bs99DypLoXwzlG2EPx7krirSsuCEX0DfidD1IKtuMu1CLJNCLlAQNV8ITGqg3NkiciywDLhOVQvqFxCRy4DLAPr1sydYTRsJpkC/SXDd526+Lkmsegc+us8liRcu98qmQUY2TP0l5B1mbRImYcWsTUFEzgVOUtXvefMXARNV9UdRZboB5apaKSKXA+ep6vFNbdfaFEzcCNe6h+jW/A/e/q1rl9Baty4QgtSOkJoFJ/0Geo2GLgPtNljjG9/bFHBXBn2j5vOADdEFVDV6EN+/An+IYTzGtK5AEHqMcNPE77uuOEqWwowLoaoMKsuhtACevdgrH4IhJ0H/I93VRO9DICXd32Mwpp5YJoU5wBARGYi7u2ga8K3oAiLSW1U3erOnAUtiGI8xsRUIuARx9dw9y6p3u76aZn7Pe/J6Nix92VspkNYRJl0O/Y9yScIasI3PYpYUVLVGRK4CXsPdkvqoqi4SkVuBfFWdBVwtIqcBNcBW4JJYxWOML1IyoM+4vRNF2WZYnw+v3AgVpa7/Jm5364JpLlEc+SPIPRRyJ9hodKZN2cNrxvitotQliY0L4cN7oaocaircukAIQhmQ3smNJ9F3ko1IZ/ZLXDy8FguWFExS2LUVCvNh3Ycw52+u6gnvbzW7v7uK6DnKTb3GQOdcX8M18c+SgjHtSXUFbJgHhXPgg7tdI3ZdFx0AWb29RHEw9By5524nuy3WeCwpGNPeVZRC0RLYON+1S9QlCg279WmdXXKou0Mqb4JLGoGgv3EbX8TDLanGmFhK7+Qerus3CSb9wC2r2uXudtr0OWxcAJsWwoIZ7hZZcL3C9p0E/Y9wdzz1GWeDEZm9WFIwpj1J7QC5491UR9X1Dls4x/UOu+YDWPGGt1JcNx15h7kE0WecddmR5CwpGNPeibg+nbr0h9HnuGU7S9w4EwWfwLpP4NO/Rg1IFIQBx7jnJnqNcVVO3QbZeBNJwpKCMckoszuM+KabwPXrVLQENnzmGrQ/fxZWvUvkjqdAiksKqZlw7E3uSqT7ENdzrGlXrKHZGNOwmirXt9PmL1zC+Ozv7hmK2uo9ZQIp7gG90edAzgjIGeYatTNzrAoqzlhDszHmwIRSodfBbgI48VeufWJHoWvE3roStqyA4mUuYYRr97w3EIK8idBjuHuWosco9zqjiz/HYlrMkoIxpuVEILuvm6KpQtkmd+dT3fTF867NQqOSRTDFDVg0ZppLEj1GuSuL9E5texymUVZ9ZIyJHVUoXQ+bF0Hx0qiksdRVRdUJprnxsXuOcg3cvcdCdj+rgmpFVn1kjPGfCHTOc9PQk/YsD4dhRwEULXYJo2gxLP0vLHuNSOO2BKHPWOg+1E05w1zSyO5vySKGLCkYY9peILDnNtlhp+xZXl3hksTGeVD0JXz+jHsQr7Yq6r1BN4DR+G+7K4oeI6DrIBubopVY9ZExJv5V7ICS5Xue1N443yWP6GQRSochX4PeY6DHSMgZDl0GWLceHqs+Msa0H+mdXd9NeVHfaTVVrn2iZJm7C6poievWY8msqDeKe/Bu1xbXxcfZf3NVUvYgXqMsKRhjElMo1V0V9B6z9/KKUndVUfwlbFkOW1fB8jdg91r421RAvNtkR7o7oHJGuJ5lO/ezMbSxpGCMaW/SO0HeoW6KtrME1rzvntj+7B8ucXz+zJ71EnR9R404DboN9h7EG+katpMoWVibgjEmeVXscLfHbl7k3QW1xF1dlG/eU0YCrkG750joOdo9zNdjZMKNp21tCsYY05z0zq6X2L4T915escM9qV202CWKosWw4GkI/3NPmQ7d3BVFjxGu08AeI1zjdmb3tj2GVmZJwRhj6kvvDH0Pc1O0ss3uDqiixa6bj5LlsPhFmPv4njKBkHcn1ImuG/LuwyBnKHQbAmkd2/Qw9oclBWOMaamsnm4aMnXPMlUo2+iuKIqXQslS2LbWPYxXU7H3+4OprpuPsRd4VxbDXG+zcdQnlCUFY4w5ECLQqY+bBp+w97qaKnf3U92dUJ88BNW74ZMH9gybCu4W2b6Hu+qnHiNcm0XPkb50TR7TpCAiJwP3AEHgEVX9fSPlzgGeBQ5TVWtFNsa0D6FUr+O/4W7+2Bvd/8O1sG2Ne8aieKn7/+JZsPbDvTsQDKW7K4vxF7nqpwFHuSqpGIrZ3UciEgSWAScChcAcYLqqLq5XLgt4GUgFrmouKdjdR8aYdquua/KixW4ci4/+AtW7IFzjnt7uOhiunrtfm46Hu48mAitUdZUX0AzgdGBxvXK/Bv4I3BDDWIwxJv5Fd00+9CQ45nq3PFzrxtlug+qkWD6RkQsURM0XessiRGQc0FdVX2pqQyJymYjki0h+cXFx60dqjDHxLBCErgPb5NmIWCaFhvq2jdRViUgAuAu4vrkNqerDqjpBVSfk5OS0YojGGGOixTIpFALRwzPlARui5rOAg4F3RGQNcDgwS0SarfMyxhgTG7FMCnOAISIyUERSgWlApPtCVd2hqt1VdYCqDgA+Bk6zu4+MMcY/MUsKqloDXAW8BiwBnlHVRSJyq4icFqv9GmOM2X8xfU5BVV8BXqm37OeNlJ0Sy1iMMcY0L3n6gzXGGNMsSwrGGGMiLCkYY4yJSLhBdkSkGFi7n2/vDpS0Yjh+sGOIH+3hOOwY4kNbHEN/VW32Qa+ESwoHQkTyW9L3RzyzY4gf7eE47BjiQzwdg1UfGWOMibCkYIwxJiLZksLDfgfQCuwY4kd7OA47hvgQN8eQVG0KxhhjmpZsVwrGGGOaYEnBGGNMRNIkBRE5WUSWisgKEbnZ73haQkT6isjbIrJERBaJyDXe8q4i8oaILPf+38XvWJsjIkERmSciL3nzA0XkE+8YnvZ60o1bIpItIjNF5Evv8zgi0T4HEbnO+3f0hYg8JSLp8f45iMijIlIkIl9ELWvwvIvzZ+9vfKGIjPcv8j0aOYbbvX9LC0Xk3yKSHbXuFu8YlorISW0db1IkBW+86PuBU4CRwHQRGelvVC1SA1yvqiNw401c6cV9MzBbVYcAs735eHcNrrfcOn8A7vKOYRvwXV+iarl7gFdVdThwCO5YEuZzEJFc4GpggqoeDARx3dnH++fwOHByvWWNnfdTgCHedBnwQBvF2JzH+eoxvAEcrKpjcGPZ3wLg/X1PA0Z57/mL9/3VZpIiKRA1XrSqVgF140XHNVXdqKqfea/LcF9EubjY/+4V+ztwhj8RtoyI5AFfBx7x5gU4HpjpFYnrYxCRTsCxwN8AVLVKVbeTYJ8DrlfkDBEJAR2AjcT556Cq7wFb6y1u7LyfDvxDnY+BbBHp3TaRNq6hY1DV173hBcCNJZPnvT4dmKGqlaq6GliB+/5qM8mSFJodLzreicgAYBzwCdBTVTeCSxxAD/8ia5G7gZuAsDffDdge9UcR75/HQUAx8JhXBfaIiGSSQJ+Dqq4H7gDW4ZLBDmAuifU51GnsvCfq3/mlwH+9174fQ7IkhSbHi453ItIReA64VlVL/Y5nX4jIN4AiVZ0bvbiBovH8eYSA8cADqjoO2EkcVxU1xKt3Px0YCPQBMnHVLfXF8+fQnET7d4WI/BRXTfyvukUNFGvTY0iWpNDceNFxS0RScAnhX6r6vLd4c91lsff/Ir/ia4GjgNO8cbhn4Kor7sZd2tcN8hTvn0chUKiqn3jzM3FJIpE+h6nAalUtVtVq4HngSBLrc6jT2HlPqL9zEbkY+AZwge55YMz3Y0iWpNDkeNHxyqt7/xuwRFXvjFo1C7jYe30x8GJbx9ZSqnqLquZ543BPA95S1QuAt4FzvGLxfgybgAIRGeYtOgFYTAJ9Drhqo8NFpIP376ruGBLmc4jS2HmfBXzbuwvpcGBHXTVTvBGRk4Ef48al3xW1ahYwTUTSRGQgrtH80zYNTlWTYgJOxbXyrwR+6nc8LYz5aNyl40JgvjediquTnw0s9/7f1e9YW3g8U4CXvNcH4f6xrwCeBdL8jq+Z2McC+d5n8QLQJdE+B+BXwJfAF8ATQFq8fw7AU7g2kGrcr+jvNnbecVUv93t/45/j7rSK12NYgWs7qPu7fjCq/E+9Y1gKnNLW8Vo3F8YYYyKSpfrIGGNMC1hSMMYYE2FJwRhjTIQlBWOMMRGWFIwxxkRYUjDGGBNhScGYFhCRsSJyatT8aa3VBbuIXCsiHVpjW8YcKHtOwZgWEJFLcA9DXRWDba/xtl2yD+8Jqmpta8dijF0pmHZFRAZ4g+D81RtQ5nURyWik7CAReVVE5orI+yIy3Ft+rjcQzQIRec/rGuVW4HwRmS8i54vIJSJyn1f+cRF5QNyASKtEZLI3sMoSEXk8an8PiEi+F9evvGVX4zqoe1tE3vaWTReRz70Y/hD1/nIRuVVEPgGOEJHfi8hib6CWO2JzRk3S8fsRcJtsas0JGIDrdXKsN/8McGEjZWcDQ7zXk3D9MoHrIiHXe53t/f8S4L6o90bmcYOozMB1s3A6UAqMxv3omhsVS113DEHgHWCMN78G6O697oPrpygH1zvrW8AZ3joFzqvbFq4bBImO0yabDnSyKwXTHq1W1fne67m4RLEXrzvyI4FnRWQ+8BBQNyDLB8DjIvJ93Bd4S/xHVRWXUDar6ueqGgYWRe3/PBH5DJiHG1mrodH/DgPeUdebaV2Xysd662pxPeaCSzwVwCMichaw6ytbMmY/hJovYkzCqYx6XQs0VH0UwA0wM7b+ClW9XEQm4UaLmy8iXynTxD7D9fYfBkJej5c3AIep6javWim9ge001J9+nQr12hFUtUZEJuJ6O50GXIXrltyYA2JXCiYpqRusaLWInAuRQd8P8V4PUtVPVPXnQAmuf/syIOsAdtkJNzjPDhHpyd4D3ERv+xNgsoh098bmnQ68W39j3pVOZ1V9BbgW14urMQfMrhRMMrsAeEBEfgak4NoFFgC3i8gQ3K/22d6ydcDNXlXT7/Z1R6q6QETm4aqTVuGqqOo8DPxXRDaq6nEicgtunAMBXlHVhsY4yAJeFJF0r9x1+xqTMQ2xW1KNMcZEWPWRMcaYCKs+Mu2eiNyPGys62j2q+pgf8RgTz6z6yBhjTIRVHxljjImwpGCMMSbCkoIxxpgISwrGGGMi/j9PmCejoHeNRgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showlogloss(cvresult=cvresult1,fromfile=False, figfilename='n_estimators_1.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 优化树参数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 树的深度：max_depth\n",
    "已取得n_estimators的最优值，其余值保持默认，调整max_depth\n",
    "\n",
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': [6, 7, 8]}\n"
     ]
    }
   ],
   "source": [
    "# 先大范围地粗调参数（步长为2），再小范围微调\n",
    "# max_depth = range(3,10,2) #7\n",
    "max_depth = [6,7,8]\n",
    "\n",
    "param2_max_depth = dict(max_depth=max_depth)\n",
    "print(param2_max_depth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, eval_metric='mlogloss', gamma=0,\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=6,\n",
       "       min_child_weight=1, missing=None, n_estimators=125, n_jobs=1,\n",
       "       nthread=-1, objective='multi:softprob', random_state=0, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.5),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_depth': [6, 7, 8]}, pre_dispatch='2*n_jobs',\n",
       "       refit=True, return_train_score='warn', scoring='neg_log_loss',\n",
       "       verbose=0)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_max_depth = XGBClassifier(\n",
    "#     max_depth=7,\n",
    "    max_depth=6,\n",
    "    learning_rate=0.1,\n",
    "    n_estimators=n_estimators,\n",
    "    objective='multi:softprob',\n",
    "    eval_metric='mlogloss',\n",
    "    booster='gbtree',\n",
    "    min_child_weight=1,\n",
    "    nthread=-1,\n",
    "    gamma=0,\n",
    "    seed=3,\n",
    "    subsample=0.5,\n",
    "    colsample_bytree=0.8,\n",
    "    colsample_bylevel=0.7)\n",
    "\n",
    "gscv2_max_depth = GridSearchCV(xgb2_max_depth,param_grid=param2_max_depth, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gscv2_max_depth.fit(X_train, y_train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/fanhong/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59174, std: 0.00345, params: {'max_depth': 6},\n",
       "  mean: -0.58976, std: 0.00412, params: {'max_depth': 7},\n",
       "  mean: -0.59183, std: 0.00294, params: {'max_depth': 8}],\n",
       " {'max_depth': 7},\n",
       " -0.5897571953084787)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gscv2_max_depth.grid_scores_,gscv2_max_depth.best_params_,gscv2_max_depth.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最优的max_depth: 7\n"
     ]
    }
   ],
   "source": [
    "# 最优的max_depth\n",
    "max_depth = gscv2_max_depth.best_params_['max_depth']\n",
    "print('最优的max_depth:',max_depth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 优化min_child_weight\n",
    "已得到n_estimators=125，max_depth=7，其余参数继续默认值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_child_weight = range(1,6,2)\n",
    "param3_mcw = dict(min_child_weight=min_child_weight)\n",
    "param3_mcw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, eval_metric='mlogloss', gamma=0,\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=7,\n",
       "       min_child_weight=1, missing=None, n_estimators=125, n_jobs=1,\n",
       "       nthread=-1, objective='multi:softprob', random_state=0, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.5),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'min_child_weight': range(1, 6, 2)},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3 = XGBClassifier(\n",
    "    max_depth=max_depth,\n",
    "    learning_rate=0.1,\n",
    "    n_estimators=n_estimators,\n",
    "    objective='multi:softprob',\n",
    "    eval_metric='mlogloss',\n",
    "    booster='gbtree',\n",
    "    min_child_weight=1,\n",
    "    nthread=-1,\n",
    "    gamma=0,\n",
    "    seed=3,\n",
    "    subsample=0.5,\n",
    "    colsample_bytree=0.8,\n",
    "    colsample_bylevel=0.7)\n",
    "\n",
    "gscv3_mcw = GridSearchCV(xgb3,param_grid=param3_mcw, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gscv3_mcw.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/fanhong/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58976, std: 0.00412, params: {'min_child_weight': 1},\n",
       "  mean: -0.58979, std: 0.00350, params: {'min_child_weight': 3},\n",
       "  mean: -0.59017, std: 0.00411, params: {'min_child_weight': 5}],\n",
       " {'min_child_weight': 1},\n",
       " -0.5897571953084787)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gscv3_mcw.grid_scores_,gscv3_mcw.best_params_,gscv3_mcw.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最佳min_child_weight： 1\n"
     ]
    }
   ],
   "source": [
    "# 最佳min_child_weight\n",
    "min_child_weight = gscv3_mcw.best_params_['min_child_weight']\n",
    "print('最佳min_child_weight：',min_child_weight)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 优化正则参数：reg_alpha&reg_lambda"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### reg_alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0, 0.1, 1, 1.5, 2]}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [0, 0.1, 1,1.5,2] \n",
    "param4_ra = dict(reg_alpha=reg_alpha)\n",
    "param4_ra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, eval_metric='mlogloss', gamma=0,\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=7,\n",
       "       min_child_weight=1, missing=None, n_estimators=125, n_jobs=1,\n",
       "       nthread=-1, objective='multi:softprob', random_state=0, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.5),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'reg_alpha': [0, 0.1, 1, 1.5, 2]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb4 = XGBClassifier(\n",
    "    max_depth=max_depth,\n",
    "    learning_rate=0.1,\n",
    "    n_estimators=n_estimators,\n",
    "    objective='multi:softprob',\n",
    "    eval_metric='mlogloss',\n",
    "    booster='gbtree',\n",
    "    min_child_weight=min_child_weight,\n",
    "    nthread=-1,\n",
    "    gamma=0,\n",
    "    seed=3,\n",
    "    subsample=0.5,\n",
    "    colsample_bytree=0.8,\n",
    "    colsample_bylevel=0.7)\n",
    "\n",
    "gscv4_ra = GridSearchCV(xgb4, param_grid = param4_ra, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gscv4_ra.fit(X_train , y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/fanhong/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58976, std: 0.00412, params: {'reg_alpha': 0},\n",
       "  mean: -0.58895, std: 0.00387, params: {'reg_alpha': 0.1},\n",
       "  mean: -0.58953, std: 0.00391, params: {'reg_alpha': 1},\n",
       "  mean: -0.58964, std: 0.00415, params: {'reg_alpha': 1.5},\n",
       "  mean: -0.59009, std: 0.00339, params: {'reg_alpha': 2}],\n",
       " {'reg_alpha': 0.1},\n",
       " -0.5889545282999017)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gscv4_ra.grid_scores_, gscv4_ra.best_params_,     gscv4_ra.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最佳reg_alpha： 0.1\n"
     ]
    }
   ],
   "source": [
    "# 最佳reg_alpha\n",
    "reg_alpha = gscv4_ra.best_params_['reg_alpha']\n",
    "print('最佳reg_alpha：',reg_alpha)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### reg_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_lambda = [0.5, 1, 2]  \n",
    "param5_rl = dict(reg_lambda=reg_lambda)\n",
    "param5_rl\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, eval_metric='mlogloss', gamma=0,\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=7,\n",
       "       min_child_weight=1, missing=None, n_estimators=125, n_jobs=1,\n",
       "       nthread=-1, objective='multi:softprob', random_state=0, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.5),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'reg_lambda': [0.5, 1, 2]}, pre_dispatch='2*n_jobs',\n",
       "       refit=True, return_train_score='warn', scoring='neg_log_loss',\n",
       "       verbose=0)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5 = XGBClassifier(\n",
    "    max_depth=max_depth,\n",
    "    learning_rate=0.1,\n",
    "    n_estimators=n_estimators,\n",
    "    objective='multi:softprob',\n",
    "    eval_metric='mlogloss',\n",
    "    booster='gbtree',\n",
    "    min_child_weight=min_child_weight,\n",
    "    nthread=-1,\n",
    "    gamma=0,\n",
    "    seed=3,\n",
    "    subsample=0.5,\n",
    "    colsample_bytree=0.8,\n",
    "    colsample_bylevel=0.7)\n",
    "\n",
    "gscv5_rl = GridSearchCV(xgb4, param_grid = param5_rl, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gscv5_rl.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/fanhong/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58988, std: 0.00415, params: {'reg_lambda': 0.5},\n",
       "  mean: -0.58976, std: 0.00412, params: {'reg_lambda': 1},\n",
       "  mean: -0.58917, std: 0.00373, params: {'reg_lambda': 2}],\n",
       " {'reg_lambda': 2},\n",
       " -0.5891705637677062)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gscv5_rl.grid_scores_, gscv5_rl.best_params_,     gscv5_rl.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最佳reg_lambda： 2\n"
     ]
    }
   ],
   "source": [
    "# 最佳reg_lambda\n",
    "reg_lambda = gscv5_rl.best_params_['reg_lambda']\n",
    "print('最佳reg_lambda：',reg_lambda)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 重新调整弱学习器数目n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimarors: 2000\n",
      "优化后n_estimators: 797\n",
      "logloss of train is: 0.4532429022373494\n"
     ]
    }
   ],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb6 = XGBClassifier(\n",
    "        max_depth=max_depth,\n",
    "        learning_rate =0.02,\n",
    "        n_estimators=2000,\n",
    "        objective='multi:softprob',\n",
    "        eval_metric='mlogloss',\n",
    "        booster='gbtree',\n",
    "        min_child_weight=min_child_weight,\n",
    "        nthread=-1,\n",
    "        gamma=0,\n",
    "        seed=3,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7)\n",
    "\n",
    "kfold6 = StratifiedKFold(n_splits=3, shuffle=True)\n",
    "cvresult6 = modelfit(xgb6, X_train, y_train, cv_folds=kfold6, filename='6_nestimators.csv',predict=True)\n",
    "# 最佳reg_lambda\n",
    "n_estimators = cvresult6.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'base_score': 0.5,\n",
       " 'booster': 'gbtree',\n",
       " 'colsample_bylevel': 0.7,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'gamma': 0,\n",
       " 'learning_rate': 0.02,\n",
       " 'max_delta_step': 0,\n",
       " 'max_depth': 7,\n",
       " 'min_child_weight': 1,\n",
       " 'missing': None,\n",
       " 'n_estimators': 797,\n",
       " 'objective': 'multi:softprob',\n",
       " 'reg_alpha': 0,\n",
       " 'reg_lambda': 1,\n",
       " 'scale_pos_weight': 1,\n",
       " 'seed': 3,\n",
       " 'silent': 1,\n",
       " 'subsample': 0.5,\n",
       " 'eval_metric': 'mlogloss'}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb6.get_xgb_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcHFW5//HP0z1bMksmy2TfIQQihBDC5gIoqIDXgIIICop6RbzigteLcK8/9XLFBfcF5aICXkUQURERBWURUSAkYU1CwmSfbDNknyGZrZ/fH6dmpmcyS2fpqZ7p7/v1qtdUnTpd9fQy/XSdU1XH3B0RERGARNwBiIhI7lBSEBGRdkoKIiLSTklBRETaKSmIiEg7JQUREWmnpCCSxsz+08x+EnccInFRUhhgzKzMzNaY2XvSysrNbJ2ZXZBWNs/M7jOz7Wa2w8yWmtn1ZjY8Wn+ZmbWaWX00rTKzj2Y59tPNrCab+9gf3cXj7l9293/N0v7WmNmZ2dh2NvTX+zXQXpfBTklhgHH3euBy4LtmVhUV3wAsdPe7AczstcCjwD+AI929EjgLaAGOTdvcE+5e5u5lwAXADWZ2XP88E9kfZlYQdwySJ9xd0wCcgNuAO4DTga3AuLR1jwPf7+PxlwGPdylbALwnbXk+sATYQUgyR6WtOyoq2xHVmZ+27hxgKbAb2AB8BigF9gApoD6axvfwvG4E/hg9/ingsAxejyOBvwDbgOXAhQcSD/BF4BfR46YCDnwAWA9sB64ATgCej577D9L2cxjwcPR+vALcDlRG634e7WtPtK+rM3iN1wCfjfbVCBREyxui57IcOKOb1+JkYDOQTCt7B/B8NH8isBDYBWwBvtXDa3o6UNPDumHA/wF1wFrgc0AiWpcEvhm9BquBK6PXsaCHba0Bzuxh3YeB6uh9vbftMwMY8G2gFtgZvUZH9/R+x/3/OpCm2APQdIBvHAwHNkX/eB9IKy8FWoHT+3j8ZaQlheiLbgdwRLR8BNAAvBkoBK6O/jmLouVq4D+j5TdF/4Azo8duAt6QFufcaL7HL5m0OG6LvgBOjL4Ebwfu7OMxpYQv7Q9Ej5kbvS6v2d946D4p3ASUAG8B9gL3AKOBCdGX0mlR/cOj16sYqAIeA76Ttu1OX369vcZp9Z8FJgFDgJnR8xyfFl+3CRNYCbw5bfnXwDXR/BPApdF8GXByD9vo8f0iJITfA+VRHCuAD0XrriB8KU+MXu+/cgBJIfpcvRK9n8XA94HHonVvBRYBlYQEcRTRD6Oe3m9NmU1qPhqg3H074RfmUOC3aauGE5oFN7cVmNkNUb9Cg5l9Lq3uyVF5PeEo4efAy9G6dwN/dPe/uHsz8A3CF9NrCb9Ey4CvunuTuz8M3AdcHD22GZhlZhXuvt3dF+/n0/utuy9w9xZCUpjTR/1/Ada4+63u3hLt7zeEJrFDEc//uPted3+Q8CV+h7vXuvsG4O/AcQDuXh29Xo3uXgd8Czitl+329hq3+Z67r3f3PYRkXxw9l0J3X+PuK3vY9h1E74eZlRN+Pd+R9nocbmaj3L3e3Z/cnxfDzJJR7Ne6+253X0M4Mrg0qnIh8F13r4k+p1/dn+2neS9wi7svdvdG4FrgFDObGj2HcsIRorn7MnfflPb8Dub9zmtKCgOUmV1C+IX2V+Braau2E5opxrUVuPvVHvoVfkf4Jd3mSXev9NCnMBZ4DfDlaN14QrNA2zZShF+pE6J166OyNmujdQDnE76E1prZ38zslP18epvT5l8lJKDeTAFOihLcDjPbQfhCGXuI4tmSNr+nm+UyADMbbWZ3mtkGM9sF/AIY1ct2e3uN26xPW18NfIpwNFMb7Wt8D9v+JfBOMysG3gksdve2fX2IcJTykpk9bWb/0kuM3RlFOEJcm1aW/v6PT4+7y/z+6Pr61BOa5iZEP0R+QGhq3GJmN5tZRVT1YN/vvKakMACZ2WhCe+qHgY8AF5rZqQDu3kBoh3/n/mzT3bcQfl2/PSraSPiybdunEZoxNkTrJplZ+udncrQOd3/a3c8lNLHcA9zVtpv9iWk/rAf+FiW4tqnM3T/az/F8JdrmbHevAC4hNG206bq/3l7jbh/j7r9099dHj3M6/yBIr7eU8IV6NvAeQpJoW/eyu19MeD2+BtxtZqWZP01eIfwan5JW1v7+E5pvJqatm7Qf207X9fUpBUbS8Tn7nrsfT/gxcwTwH1F5T++3ZEBJYWD6AXCPuz8SHTJfDfw4+lVItPxBM7smSiCY2URgWk8bNLORhM7IJVHRXcDbzOwMMysE/p3Q2flPQtJpAK42s0IzO52QTO40syIze6+ZDYuaRHYRmj0g/MIeaWbDDtHr0OY+4AgzuzSKp9DMTjCzo/o5nnJCJ/IOM5tA9CWVZgswPW25t9d4H2Y208zeFL3PewlHKa3d1Y38EvgEcCqhT6FtO5eYWVV0ZLIjKu5xO2ZWkj4RjkTvAq6PToeeAnyacGTU9rw+aWYTzKyS0Dnel8Iu+ymI4v+Amc2JnvOXgafcfU30/p4UvW4N0evR2sf7LZmIu1ND0/5NwHmEX1CVXcofAq5PWz4JuJ/wT78DeBG4HhgZrb+M8M/SduZNLaHNeXTaNt5B6DDcCfyNqOM2WveaqGxnVOcdUXkR8GdCM9Yu4Gng9WmPu4XQBLCDns8++lLa8un00Tkd1ZtJOGOpLtr+w4S+iP2Kh+47mgvS6teQ1olP+CL8XNprsih6PZ8lfMnXpNU9F1gX7eszGbzGa+jcMT2b0Pezm9AZf193r2Fa/cmEL/A/din/RfR+1xN+BJzXw+NPj55/1+lwQt/VL6LXez3weTrOPiogHMluJZx9dBXhyMJ62M+abvbxpWjdFYRO87bnOzEqP4NwxlE9HWd6lfX1fmvqe7LoBRYRyQozOxu4yd2n9FlZYqfmIxE5pMxsiJmdY2YFUTPaFwgnOcgAoCMFGRDM7A3An7pb5+HsKckRZjaU0BR2JKHf44/AJ919V6yBSUaUFEREpJ2aj0REpN2Au8nWqFGjfOrUqXGHISIyoCxatOgVd6/qq96ASwpTp05l4cKFcYchIjKgmNnavmup+UhERNJkLSmY2S1mVmtmL/aw/kgze8LMGs3sM9mKQ0REMpfNI4XbCAO79GQb4RL8b2QxBhER2Q9ZSwru/hjhi7+n9bXu/jTh8ncREckBA6JPwcwuN7OFZrawrq4u7nBERAatAZEU3P1md5/n7vOqqvo8o0pERA7QgEgKIiLSP/ImKdSsX8sj9/6M+nrdfkVEpCdZu3jNzO4g3I99lJnVEO6UWAjg7jeZ2VhgIVABpMzsU8CsbN00q+7Fh3jj4quonn4Uhx99YjZ2ISIy4GUtKXgY7q+39ZvpPGRfVpVWTQZg95a1oKQgItKtvGk+GjF2KgB7tx3oGOIiIoNf3iSF4WMmk3IjtWND35VFRPJU3iSFZGER26ySZP2muEMREclZeZMUAHYUjKJk75a4wxARyVl5lRQaSsZQ0VQbdxgiIjkrr5JC89CxjExtRUOQioh0L6+SglWMZ5g1sHV7j/fpExHJa3mVFApHhMsitm3KaAAiEZG8k1dJoXRUuIBtV62SgohId/IqKVRGF7Dt2VoTbyAiIjkqr5LC8DFTAGjdoaQgItKdvEoKieKh7KBcF7CJiPQgr5ICwA7KKNnxctxhiIjkpLxLCg3l06m0hrjDEBHJSXmXFJrLJzEmVcveppa4QxERyTl5lxSSwydRYXvYtGVz3KGIiOScvEsKQ0ZPB2BrjfoVRES6yrukUDnucADqa1fHHImISO7Ju6QwYkJICi1b18QbiIhIDsq7pJAYOpwGhpDYpWE5RUS6yrukgBlbC8cytGFj3JGIiOScrCUFM7vFzGrN7MUe1puZfc/Mqs3seTObm61YumoYMp7hzbqqWUSkq2weKdwGnNXL+rOBGdF0OfCjLMbSSUv5RMZ5Hbv3NvfXLkVEBoSsJQV3fwzobTSbc4H/8+BJoNLMxmUrnnTJEVOosFfZuFnXKoiIpIuzT2ECkN7bWxOV7cPMLjezhWa2sK6u7qB3PHT0NAC2bag+6G2JiAwmcSYF66as28GT3f1md5/n7vOqqqoOesfDx88AoGHLqoPelojIYBJnUqgBJqUtTwT65ZSgirHhqubmbRqBTUQkXZxJ4V7gfdFZSCcDO929X04JsqEjeJUhJHeu64/diYgMGAXZ2rCZ3QGcDowysxrgC0AhgLvfBNwPnANUA68CH8hWLN0Ex7aicZS+qhHYRETSZS0puPvFfax34GPZ2n9fXi2bwuitK2huTVGYzL9r+EREupO334Y+fBqT2cL6V3bHHYqISM7I26RQMnYGxdbCpvUr4w5FRCRn5G1SGDnpKAB2blgecyQiIrkjb5NC2bgjAGh64Z6YIxERyR15mxQoH0cjRRSUlMUdiYhIzsjfpJBIsK14AuWv6loFEZE2+ZsUCKeljm3ZyJ6m1rhDERHJCXmdFBgxnSm2hVV1u+KOREQkJ+R1UigbdwQl1kzNWp2WKiICeZ4URkx+DQA71i2JORIRkdyQ10mhcGy4ViFV+1LMkYiI5Ia8TgqUjqI+UcHQXWo+EhGBfE8KZuwqm8bYprU6A0lEhHxPCkBL46scbhuorq2POxQRkdjlfVIYMu89jLTdrFmni9hERPI+KYyYfDQAO9a/GHMkIiLxy/ukkBxzJABrli2OORIRkfhlbeS1AaNiIo1WwszExrgjERGJXd4fKZBIsNMqGNu0lvrGlrijERGJlZIC0DrpZA5LbGTZJt0DSUTyW1aTgpmdZWbLzazazK7pZv0UM3vIzJ43s0fNbGI24+lJ2falTLCtrFi7IY7di4jkjKwlBTNLAjcCZwOzgIvNbFaXat8A/s/dZwPXAV/JVjy9KXvblwDYvua5OHYvIpIzsnmkcCJQ7e6r3L0JuBM4t0udWcBD0fwj3azvFzZ2dvi7+YU4di8ikjOymRQmAOvTlmuisnTPAedH8+8Ays1sZNcNmdnlZrbQzBbW1dUd+kgrxrOnoIKR9SvY26zbXYhI/spmUrBuyrzL8meA08zsGeA0YAOwzylA7n6zu89z93lVVVVZiNRoGH4UR9paVmzZfei3LyIyQGQzKdQAk9KWJwKdLgZw943u/k53Pw74r6hsZxZj6lHRhNnMtPUs3bA9jt2LiOSEbCaFp4EZZjbNzIqAi4B70yuY2Sgza4vhWuCWLMbTq/IpcxhiTWxatTSuEEREYpe1pODuLcCVwAPAMuAud19iZteZ2fyo2unAcjNbAYwBrs9WPH2xsccA0LJRZyCJSP7K6m0u3P1+4P4uZZ9Pm78buDubMWSs6khaSDJ0+zKaWlIUFei6PhHJP/rma1NQzKsVh3GUrdOVzSKSt5QU0hSMn82sxFoWr1Nns4jkJyWFNEOnzGWsbeflVRqzWUTyk5JCuvFzAWhdvyjmQERE4qGkkG7cbFKWZMKrS6ndvTfuaERE+p2SQrqiUvYmy5hjK1m8dkfc0YiI9DslhS6KjzmX2YlVPLN2W9yhiIj0OyWFLpJr/0mlNbBxta5sFpH8o6TQ1YW3AVC45RndMVVE8o6SQldVR9FKktf4Sp5Zp34FEckvSgpdJQvwCfOYk6jmyVVb445GRKRfKSl0o2DyCRyTWMPClZviDkVEpF8pKXRn8ikU0UzrugXqVxCRvKKk0J3JJwMwx6rVryAieUVJoTulo2gtGMoJieXqVxCRvKKk0IPkMedzYsHLPFldF3coIiL9RkmhJ1NeS7nXs6vmRXbtbY47GhGRfqGk0JOoX2EuL/GPl1+JORgRkf7RZ1Iws8PMrDiaP93MPmFmldkPLWbDp+FlYzghsZzP/35J3NGIiPSLTI4UfgO0mtnhwE+BacAvsxpVLjDDUq2clnyBpIG7xx2RiEjWZZIUUu7eArwD+I67XwWMy25YOaKkkuHsorR+FS9t3h13NCIiWZdJUmg2s4uB9wP3RWWFmWzczM4ys+VmVm1m13SzfrKZPWJmz5jZ82Z2Tuah94NL7gbgdYkXeXS5zkISkcEvk6TwAeAU4Hp3X21m04Bf9PUgM0sCNwJnA7OAi81sVpdqnwPucvfjgIuAH+5P8Fk3YhoUlHB2wSIeWrYl7mhERLKuoK8K7r4U+ASAmQ0Hyt39qxls+0Sg2t1XRY+9EzgXSB+owIGKaH4YsDHz0PtJSSXHt6zg2XWvsGXXXsZUlMQdkYhI1mRy9tGjZlZhZiOA54BbzexbGWx7ArA+bbkmKkv3ReASM6sB7gc+3kMMl5vZQjNbWFfXz804Z3+VIpqYzUoeWLK5f/ctItLPMmk+Gubuu4B3Are6+/HAmRk8zrop63oKz8XAbe4+ETgH+LmZ7ROTu9/s7vPcfV5VVVUGuz6Epp0GGPMrVnD/C7prqogMbpkkhQIzGwdcSEdHcyZqgElpyxPZt3noQ8BdAO7+BFACjNqPfWTf0BFQVMpbmh9hwept1O1ujDsiEZGsySQpXAc8AKx096fNbDrwcgaPexqYYWbTzKyI0JF8b5c664AzAMzsKEJSyL3TfIaMYFxqE8N9Jw8uVROSiAxefSYFd/+1u892949Gy6vc/fwMHtcCXElIKMsIZxktMbPrzGx+VO3fgQ+b2XPAHcBlnotXiV10Owa8e9hS/vSCkoKIDF59nn1kZhOB7wOvI/QJPA580t1r+nqsu99P6EBOL/t82vzSaLu5bewxUDGRdySf56aVp1C7ay+jdRaSiAxCmTQf3Upo9hlPOHvoD1FZ/jADb+WwHY9T6E2c/6N/xh2RiEhWZJIUqtz9VndviabbgH4+BSgHDBlBwls5o/gl6uobdS8kERmUMkkKr5jZJWaWjKZLgPwbjuzyR8CSXGF3s7c5xbPrNUyniAw+mSSFDxJOR90MbAIuINz6Ir8UFMNRb+fo0l0UWIqP/HxR3BGJiBxymZx9tM7d57t7lbuPdvfzCBey5Z9Z80k01PKmIdVsbWhib3Nr3BGJiBxSBzry2qcPaRQDxRFnQ2Ep1x32Eq0p55zv/j3uiEREDqkDTQrd3cJi8CsaCkVDGbPybo6sKqa0uEAdziIyqBxoUsjfb8LSKizVwmdnbOCFDTtZtHZ73BGJiBwyPSYFM9ttZru6mXYTrlnITx95DBIFvOGlL1FRUsCt/1wTd0QiIodMj1c0u3t5fwYyYCQLYegoChpqufT4Udz0xGY27NjDhMohcUcmInLQDrT5KL+961bwFB9a+SkMuPlvK+OOSETkkFBSOBCTToZkMSP2rmP40CL+78m11O7aG3dUIiIHTUnhQCQSUDYG9u7gnveMJ2HGzY+tijsqEZGDpqRwoD70AAAT7nkX5x47ntufWqcBeERkwMtkjObuzkJab2a/iwbcyU8V42HICKjfwsdPn8ze5lbe9j1dzCYiA1smRwrfAv6DcNvsicBngB8DdwK3ZC+0AaB8HKSamVb7MJeeMoWtDU1U19bHHZWIyAHLJCmc5e7/6+673X2Xu98MnOPuvwKGZzm+3HbF41BQAvddxSfPmMGQwiRf+/NLcUclInLAMkkKKTO70MwS0XRh2rr8vbIZog7nsdC4i5G7lvLR0w/jL0u38OSq/LuzuIgMDpkkhfcClwK10XQpcImZDSGMwZzfrngMiivg8e/wwddNo6ggwWW3LqCxRXdQFZGBJ5NbZ69y97e7+6hoeru7V7v7Hnd/vD+CzGklw0JSWHoPQ3av4eZLj2dvc4obH9EFbSIy8GRy9tHE6EyjWjPbYma/MbOJmWzczM4ys+VmVm1m13Sz/ttm9mw0rTCzgTmc2YcfBgxuPYfTZ45mZGkR33/oZVZs2R13ZCIi+yWT5qNbgXsJN8GbAPwhKuuVmSWBG4GzgVnAxWY2K72Ou1/l7nPcfQ7wfeC3+xd+jigfEy5mq98Cu7fw4FWnkkwY7/zhP2lN5Xe3i4gMLJkkhSp3v9XdW6LpNqAqg8edCFRHzU9NhFNYz+2l/sXAHRlsNzd94H7A4cdvZGRZMV9/12zqG1v44SPVcUcmIpKxTJLCK2Z2iZklo+kSIJPTayYA69OWa6KyfZjZFGAa8HAG281NIw+DoVWwexPs3sJ5cyZw3pzxfPuvK1iwelvc0YmIZCSTpPBB4EJgM7AJuAD4QAaP6250tp7aUi4C7nb3bk/ZMbPLzWyhmS2sq6vLYNcx+dAD4A43n4aZ8aV3HENhMsElP3mKHa82xR2diEifMjn7aJ27z3f3Kncf7e7nAe/MYNs1wKS05YnAxh7qXkQvTUfufrO7z3P3eVVVmbRcxWTkYZAoCEcL21ZTVlzA3Ve8lubWFKfe8Agtram4IxQR6dWB3hDv0xnUeRqYYWbTzKyI8MV/b9dKZjaTcGX0EwcYS2751AtgCfjpmwE4ZuIwpo4qZdfeFl3tLCI570CTQndNQ524ewvh4rYHgGXAXe6+xMyuM7P5aVUvBu5098Fxmk7FOCgfDw11sP5pAB75zOm8/5Qp/Pjvqzn964/EHKCISM/sQL6LzWydu0/OQjx9mjdvni9cuDCOXWeusR5umBaG7rxmPSSSNLemOOFLf2XHnmZ++N65nHPMuLijFJE8YmaL3H1eX/V6PFLo4ZbZu8xsN+GaBelJcRkMnwZNDfCDEwAoTCZ44tozKCsu4GO3L+axFTncYS4ieavHpODu5e5e0c1U7u4F/RnkgPSxp8ItMHashYZXABhSlOQfn30TQ4qSvP/WBTz80paYgxQR6Uwjr2WLGXzwQUi1wg9PaS8eNrSQf3z2TRjwwdsW8qcXNsUXo4hIF0oK2TT6SBg2ERpq4caT24uHlxbx7BfeQllxAR+9fTGn3vAwg6WfXUQGNiWFbPv4Yigsha3V0NBxIXhFSSFP/9eZjCgtYt22PZx4/V9patF1DCISLyWFbCsogg89CKkWuPGEcMVzZEhRkoX/dSYTKkuoq2/iuP95kPXbXo0xWBHJd0oK/WHs0VA5GV7d2n42UptEwvjHNWdw0yVzaWhs5Q03PMIfn1c/g4jEQ0mhv3ziGbAkbH0ZNr+wz+qzjh7H369+IwmDj/1yMR+/4xnqdjfGEKiI5DMlhf6SSMK/LwcMbnoD7Nl3PKFJI4ay/Etnc9WZR/CH5zZywvV/5a6F69UJLSL9RkmhP5VVwZhjAIfvHQepfTuWC5MJPnnmDP766VMpLy7g6ruf55gvPsiitdv7P14RyTtKCv3to3+H4dNhzzb4ztE9Vjt8dDnPfeEtfOWdx9DQ2ML5P/onH/n5QlbW1fdjsCKSbw7o3kdxGhD3PuqLO3zrqHCL7RGHwScW91q9obGFWx5fzTf/sgKAdxw3gQ+9fhpHTxjWH9GKyCCQ6b2PlBTi0toC3zgc9myHqiPDbTH6sLW+kR8+upKfPr4agPKSAr7z7jm8ceZoEok+b1wrInlMSWEgaKyHr04Gb4X3/gZmnJnRw3btbebOBev4yv0v4YQ7anzmLTM577gJTKgckt2YRWRAUlIYKPbsCLfZ9hS8716YflrGD21uTfGnFzdz1Z3P0Bq9jeUlBXzmLTM56+ixjKkoyVLQIjLQKCkMJA1bQ1OSp2D0LPi3/R+Ebv22V/ndMxv4zl9XkIre0rLiAq568xGcdfRYHUGI5DklhYEmPTFc+HOYNb/vx/SgunY3f3phM99OSxAJgw+/YTqnHlHFcZMrGVqku5+L5BMlhYFozw64YXroYzjra3DSR0KHwUFY/UoDf1m6ma/9eTmtqY73OmHw3pOmMHdKJXMnD2fyiKHYQe5LRHKXksJA1VgPX5sSbqBXOho+9QIUHpq+gYbGFhas3sbiddv5/sPVndYZcMZRozlu8nDmTh7OsZOG6WhCZBBRUhjIUin47mzYuR4sAVctgYpDPwJqa8pZsWU3i9dt5wu/X0JLqvNnIWFw0YmTObyqjOlVpRxWVcb4yiEkdfqryICjpDAYLPsD/OqSMP/BB2HySVnf5faGJp5Zv53Fa3fwo7+t7NTk1GZIYZI3HlnF9FEhWUyPkkZFSWHW4xORA5MTScHMzgK+CySBn7j7V7upcyHwRcCB59z9Pb1tM6+SAkDtMvhhNGrb2TfAiZcfdD/D/nB3tjY0sbK2nlWvNLCqrp4f/311j/VPmjaC6VVlHFZVytSRpYyrLGHcsCEMH1qoPguRGMWeFMwsCawA3gzUAE8DF7v70rQ6M4C7gDe5+3YzG+3utb1tN++SAoSrnr9+eOhnsCRc9WJWmpP2V1NLinXbGlhZ18CqupAwfr2optfHJBPG/GPHM25YCeOGlTB22BDGDSthTEUJI0uLdGW2SJbkQlI4Bfiiu781Wr4WwN2/klbnBmCFu/8k0+3mZVKAcL+k7x8P21aG5VEz4coF8cbUi20NTazd2sDmnXvZtHMvm3ft5e5FNWxraMp4G287ZhzDSwsZUVrMyNIiRpQWMbK0iOFpfwuTuqejSCZyISlcAJzl7v8aLV8KnOTuV6bVuYdwNPE6QhPTF939z91s63LgcoDJkycfv3bt2qzEPCC8Uh0N65mCREEYA3r4lLij2m+tKWdrQ2NH0ti5l7rdjWxtaOKOBesOeLtVZUWcd9wEhg0pZNjQIoYNKaS8uIDykgLKSgooKy6gvKSQsuICdZhLXsmFpPAu4K1dksKJ7v7xtDr3Ac3AhcBE4O/A0e6+7wg0kbw9UkjX2hKOGnasCcuVU+DKp6GgONawsqWlNcWOPc1sa2hia30T219tYmtDE9vqm9jW0MjtT63b58ypg2HAJSdPobS4gLLiJEOLCigpTDKkKMGQwiTFhUmGFCZDWft8gpKiJCUFSQqTpv4TyTmZJoVsnoheA0xKW54IbOymzpPu3gysNrPlwAxC/4P0JFkAn3oOdtbAd4+FHWvh+rHw7tth5tn92hHdHwqSCUaVFTOqrBjG7Lv+v8/dd1yKxpZWdu5pZteeZnbvbaG+sYX6vS3s3tvC7vb5Zn72xBqaWzsnFAd+/mR2j0YNuHDepPZkkp5kSgoTlBQmKS5IUlyYoDiZCH8LkhQVJCgu6DxfVJCgIKFEJIdGNo8UCghNQ2cAGwhf9O9x9yVpdc4idD6/38xGAc8Ac9x9a0/b1ZFCN6ofgtsvCE0JznduAAASoklEQVRKloQPPwzj58Qd1aDQ2NLKnqZW9jan2NMczbe0sjf6u6cplO9Nm7754AoG1oneHQqTxgdfP42iZIKCRIKCpJEwoyBhJBJG0sLJAslEgmQCEmbRcjRZqNdRv/P6tvoFafPJBGF7ZiQSdNpWd49t24fsn9ibj6IgzgG+Q+gvuMXdrzez64CF7n6vhZ823wTOAlqB6939zt62qaTQg9ZmWHQb3P8fgMPQUXDZH2H0kXFHJhloTXlHAmpJsaeplcaWVppaUjS2pNr/7lvWefmeZzewZVdj3E9HuhhbUcL8OeMxC4nUiP4aWNpywugos7QywvKxkyo5YeqIA4ohJ5JCNigp9GHvTvjHd+Hv3wzLR58Pp30WqmbGG5cMOC2tKVrdSaWg1Z3WVg9/U07KnZaUk0qF5bbytqm39Sl3WlPQmkqFv+7t86mobvpjb/vnatxhy+5Gmlr2Hdc8n4wbVsIT155xQI9VUsh3DVvhie/D498Oy0Or4H33wNiex4UWkXi4O+6Qcifl4ITljjLHgaJk6G86EEoKEjS8Av/8Xjh6ACiphPN/CoefMeg6pEWkZ5kmBV35M9iVjoI3XwdXr4YzvhCal24/H64bAQtvhcbdcUcoIjlERwr5pqUJlvwW7vloOFsJ4LhLYO77YeIJOnoQGaTUfCS9c4eahfCr90L9llBmCXjLl2D2RVA6Mt74ROSQUlKQzDXuhiW/g3s/3lE26zyY+z6Y/kZIqJVRZKBTUpADs2Up/OJ82N128bnB6dfCnPdA5aReHyoiuUtJQQ5OSyO8dB/85sNhzGiA4go44/PhKKKsKt74RGS/KCnIobN9DTz/a3j0yx2d08XD4Iz/B0e9HcrHxhqeiPRNSUEOPXeoXQov/hYe/1ZagqiAN/5nSBDDJsYbo4h0S0lBsq/2JVj6e/jbVzsShCXCbTVmngNjj9EpriI5QklB+tcr1aEP4qHrOvogMDj+snD19LTToKQizghF8pqSgsSnvhZWPAB/+Tzs2dZRXlwBr78KZrwZxhytowiRfqSkILmhpQlqFkD1X+Gpm6G5oWPdcZeGo4ipp+piOZEsU1KQ3LRrU0gQ1X8J/RFtLAEn/xtMOxWmvBaKy+OLUWQQUlKQ3NfaDBufhdWPwj+/H27W16aoHE74IEx5PUw+CUqGxRamyGCgpCADT/MeWL8AVv8NFvwYGnd1rLMEnHQFTD4FJp8MZaPji1NkAFJSkIGv6VXYsBDW/AOe/BE0ph1JYDD73TDpRJh0Eow+ChIHNviISD5QUpDBp6UxNDfVLID1T8GyP3ReP+1UmHhiSBQTT4ChBzaWrchglGlSKOiPYEQOiYLi0L8w+STg4+EK6x1rQ5PT+gXw3C9h9WMd9S0RjiYmHA/j58KY10BhSWzhiwwEOlKQwaWpIe1o4mlYfj+Q9hm3RDgVdsJcGH8cjJ4FycLYwhXpLzlxpGBmZwHfBZLAT9z9q13WXwZ8HdgQFf3A3X+SzZhkkCsqhamvCxOEo4mdNbDxmY5p8c/C1GbC8SFBjI8SRdVM9U9I3spaUjCzJHAj8GagBnjazO5196Vdqv7K3a/MVhyS58zCOBCVk2DW/FDmHu78unFxSBKLfgYbFnV+3KSTYdyxHVPVTB1RSF7I5pHCiUC1u68CMLM7gXOBrklBpH+ZwYhpYTr6/DAEaSoF21bChsUdRxQL/rfz48YfB2Nnw7jZMPbY0EdRNDSe5yCSJdlMChOA9WnLNcBJ3dQ738xOBVYAV7n7+q4VzOxy4HKAyZMnZyFUyXuJBIyaEaZj3x3KUq2wbRVsei4kic3Pd252Ahh1BIybA+PndCSN4rL+j1/kEMlmUujubmdde7X/ANzh7o1mdgXwM+BN+zzI/WbgZggdzYc6UJFuJZIdieKYC0KZO+xcD5tfgE3Ph0Sx9B544a6Oxw2fFo4iRs8Kf8e8BkZMVz+FDAjZTAo1QPqgvhOBjekV3H1r2uKPga9lMR6Rg2cGlZPDdOTbOsp3b4FNz4ajii1Lwl1iX7qv82PHzelIEqNnhTvFalhTyTHZTApPAzPMbBrh7KKLgPekVzCzce6+KVqcDyzLYjwi2VM+BsrfCke8taOseQ/ULQ+j1W1ZEqbn74JUc0ed0qooSbwGxkRHFlVHQuGQ/n8OImQxKbh7i5ldCTxAOCX1FndfYmbXAQvd/V7gE2Y2H2gBtgGXZSsekX5XOCTqa5jTuby+DmqXwJYoWdQugSdv7FynYAgc8ZYoWUQJo3Jq6PsQySJdvCaSC1KtsG11lCzSpu2rO9ebMK9zE9ToWRqLQjKSExeviUiGEkkYdXiYZp3bUd7UEMbC3vJiRzPUs7dDqiXtsYVhDIrRR4XrKaqOhFEzlSzkgCgpiOSyolKYeHyY2rjD7s0hSdQui6Yl8NRNXR5sMOV1HYmi6ojwt2yMhkKVHikpiAw0ZlAxLkyHn9FRnkrBrhqoWwF1L0XTclj40323MfFEGHk4jDys4++I6SEJSV5TUhAZLBKJjtNlZ5zZUe4O9Vs6kkTdSyFxvHg3tDZ13kb5+JAcRk4Pf0ccFhLG8Gm6ejtPKCmIDHZmUD42TNNP77yusT5ctb21GrauDPPbVsKzd3Q+dRaUMPKEkoJIPisuC/dyGjd733V7d4YzorathK2rOhLG4p+zz80JlDAGDSUFEeleybDur7OAzglj26ooaShhDAZKCiKy/5QwBi0lBRE5tPY7YazKPGEMnxqmkop+eCL5SUlBRPpPrwljV0e/RXrC6K7Te8iIjgQxYlo4shgxPcyXjdXtQA6CkoKI5IaSir4TxvY10bQatq8N41ws+W3nugVDomQxvWMwpeFTQ+IYNgkKirL/XAYwJQURyX29JYzW5jDGxbbVHYmjbX7lw9Cyp3P9YZO6OcqIEseQ4dl/LjlOSUFEBrZkYXRUMB04o/O6VArqN4ejivSjjG2rw5gXDbWd6xcPCxf/DZ8ClVP2/ZsHnd9KCiIyeCUSUDE+TFNO2Xd9Y33n5qgda8PfrdVQ/dC+RxmlVSE5tF05Xjk5bXnSoBgHQ0lBRPJXcRmMPTpMXblDQ11asljTkTQ2PQvL/rBvB3hpVUei6HqUMUD6M5QURES6YwZlo8M06YR917c1Te1YF01rO+Y3PgPL7u18i3OAigndJ4zKyeFoJgfG8VZSEBE5EOlNU5NP3nd9qhV2bew4umhLHNvXwurHwrpO12ZY1AHetT9jakgapVX9cstzJQURkWxIJEM/Q+UkmPr6fde3NMLOmo5mqR3rOpqqXvojvPpK5/qFQ+H1V8FpV2c1bCUFEZE4FBRH41kc1v36xvrORxc71oVhWLMdVtb3ICIi+6+4DMbMClM/yuq14GZ2lpktN7NqM7uml3oXmJmbWZ+DSouISPZkLSmYWRK4ETgbmAVcbGb7pDwzKwc+ATyVrVhERCQz2TxSOBGodvdV7t4E3Amc2029/wFuAPZmMRYREclANpPCBGB92nJNVNbOzI4DJrn7fb1tyMwuN7OFZrawrq7u0EcqIiJAdpNCdyfUtp+Ua2YJ4NvAv/e1IXe/2d3nufu8qqqqQxiiiIiky2ZSqAEmpS1PBDamLZcDRwOPmtka4GTgXnU2i4jEJ5tJ4WlghplNM7Mi4CLg3raV7r7T3Ue5+1R3nwo8Ccx394VZjElERHqRtaTg7i3AlcADwDLgLndfYmbXmdn8bO1XREQOnLl737VyiJnVAWsP8OGjgFf6rBWPXI1Nce2fXI0Lcjc2xbV/DjSuKe7eZ6fsgEsKB8PMFrp7TvZZ5Gpsimv/5GpckLuxKa79k+24NLq1iIi0U1IQEZF2+ZYUbo47gF7kamyKa//kalyQu7Eprv2T1bjyqk9BRER6l29HCiIi0gslBRERaZc3SSHTsR2ytO9bzKzWzF5MKxthZn8xs5ejv8OjcjOz70VxPm9mc7MY1yQze8TMlpnZEjP7ZC7EZmYlZrbAzJ6L4vrvqHyamT0VxfWr6Ep5zKw4Wq6O1k/NRlxp8SXN7Bkzuy/H4lpjZi+Y2bNmtjAqy4XPWaWZ3W1mL0WftVPijsvMZkavU9u0y8w+FXdcafFdFX32XzSzO6L/if75nLn7oJ+AJLASmA4UAc8Bs/px/6cCc4EX08puAK6J5q8BvhbNnwP8iXBDwZOBp7IY1zhgbjRfDqwgjH0Ra2zR9sui+ULCWBsnA3cBF0XlNwEfjeb/Dbgpmr8I+FWW389PA78E7ouWcyWuNcCoLmW58Dn7GfCv0XwRUJkLcaXFlwQ2A1NyIS7C3aRXA0PSPl+X9dfnLKsvdq5MwCnAA2nL1wLX9nMMU+mcFJYD46L5ccDyaP5/gYu7q9cPMf4eeHMuxQYMBRYDJxGu4izo+p4SbqVySjRfENWzLMUzEXgIeBNwX/QlEXtc0T7WsG9SiPW9BCqiLzjLpbi6xPIW4B+5Ehcdww6MiD439wFv7a/PWb40H/U5tkMMxrj7JoDo7+ioPJZYo0PO4wi/ymOPLWqieRaoBf5CONLb4eGeWl333R5XtH4nMDIbcQHfAa4GUtHyyByJC8Kt6R80s0VmdnlUFvd7OR2oA26Nmtx+YmalORBXuouAO6L52ONy9w3AN4B1wCbC52YR/fQ5y5ek0OvYDjmm32M1szLgN8Cn3H1Xb1W7KctKbO7e6u5zCL/MTwSO6mXf/RKXmf0LUOvui9KL444rzevcfS5hCNyPmdmpvdTtr9gKCE2nP3L344AGQrNM3HGFnYV2+fnAr/uq2k1ZVuKK+jHOBaYB44FSwnva0/4PaWz5khT6GtshDlvMbBxA9Lc2Ku/XWM2skJAQbnf33+ZSbADuvgN4lNCOW2lmBd3suz2uaP0wYFsWwnkdMN/C+B93EpqQvpMDcQHg7hujv7XA7wjJNO73sgaocfe2MdjvJiSJuONqczaw2N23RMu5ENeZwGp3r3P3ZuC3wGvpp89ZviSFXsd2iMm9wPuj+fcT2vPbyt8Xne1wMrCz7XD2UDMzA34KLHP3b+VKbGZWZWaV0fwQwj/JMuAR4IIe4mqL9wLgYY8aWA8ld7/W3Sd6GP/jomg/7407LgAzKzWz8rZ5Qjv5i8T8Xrr7ZmC9mc2Mis4AlsYdV5qL6Wg6att/3HGtA042s6HR/2jba9Y/n7NsduDk0kQ4e2AFoW36v/p533cQ2gabCVn9Q4Q2v4eAl6O/I6K6BtwYxfkCMC+Lcb2ecJj5PPBsNJ0Td2zAbOCZKK4Xgc9H5dOBBUA14XC/OCoviZaro/XT++E9PZ2Os49ijyuK4bloWtL2GY/7vYz2NQdYGL2f9wDDcySuocBWYFhaWexxRfv7b+Cl6PP/c6C4vz5nus2FiIi0y5fmIxERyYCSgoiItFNSEBGRdkoKIiLSTklBRETaKSmIiEg7JQWRDJjZHDM7J215vh2iW7BHt2weeii2JXKwdJ2CSAbM7DLCBUtXZmHba6Jtv7Ifj0m6e+uhjkVERwoyqJjZVAsDufw4GqTkwehWGd3VPczM/hzdVfTvZnZkVP6uaHCT58zssejWKNcB744GZHm3mV1mZj+I6t9mZj+yMGDRKjM7zcLASsvM7La0/f3IzBZa54GDPkG46dkjZvZIVHaxhcFyXjSzr6U9vt7MrjOzp4BTzOyrZrbUwqAv38jOKyp5J5uXamvS1N8TYdyKFmBOtHwXcEkPdR8CZkTzJxHuGQPhNgYTovnK6O9lwA/SHtu+DNxGuEGeEe5uuQs4hvCja1FaLG23TEgSbvI3O1peQzQOAiFBrAOqCHcYfRg4L1rnwIVt2yLc09/S49Sk6WAnHSnIYLTa3Z+N5hcREkUn0e3CXwv8Ohq34X8Jg6oA/AO4zcw+TPgCz8Qf3N0JCWWLu7/g7inCfYja9n+hmS0m3NfpNYRR7ro6AXjUwx0yW4DbCSP3AbQS7mgLIfHsBX5iZu8EXs0wTpFeFfRdRWTAaUybbwW6az5KEAYtmdN1hbtfYWYnAW8DnjWzfer0ss9Ul/2ngAIzmwZ8BjjB3bdHzUol3Wynu3vjt9nrUT+Cu7eY2YmEO2heBFxJuJW3yEHRkYLkJQ+DCa02s3dB+8Dsx0bzh7n7U+7+ecLQhpOA3YRxrA9UBWGAmZ1mNobOg6akb/sp4DQzG2VmScKtnf/WdWPRkc4wd78f+BThTqQiB01HCpLP3gv8yMw+BxQS+gWeA75uZjMIv9ofisrWAddETU1f2d8duftzZvYMoTlpFaGJqs3NwJ/MbJO7v9HMriXcO9+A+9399/tukXLg92ZWEtW7an9jEumOTkkVEZF2aj4SEZF2aj6SQc/MbiSMr5zuu+5+axzxiOQyNR+JiEg7NR+JiEg7JQUREWmnpCAiIu2UFEREpN3/B7I+IhLXsd5FAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showlogloss(cvresult=cvresult6,fromfile=False, figfilename='n_estimators_6.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存模型，供测试使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#保存模型\n",
    "import _pickle as pk\n",
    "pk.dump(xgb6, open(\"xgb_model.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of train is: 0.4532429022373494\n"
     ]
    }
   ],
   "source": [
    "#保存数据\n",
    "xgb = pk.load(open(\"xgb_model.pkl\", 'rb'))\n",
    "\n",
    "train_predprob = xgb.predict_proba(X_train)\n",
    "logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "#Print model report:\n",
    "print('logloss of train is:', logloss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 调用模型进行测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2950</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3758</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3300</td>\n",
       "      <td>1650.000000</td>\n",
       "      <td>1100.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4900</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.0         1   2950      1475.000000     1475.000000        0.0   \n",
       "1        1.0         2   2850      1425.000000      950.000000       -1.0   \n",
       "2        1.0         1   3758      1879.000000     1879.000000        0.0   \n",
       "3        1.0         2   3300      1650.000000     1100.000000       -1.0   \n",
       "4        2.0         2   4900      1633.333333     1633.333333        0.0   \n",
       "\n",
       "   room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  water  \\\n",
       "0       2.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "1       3.0  2016      6   24  ...         0     0      0    1       0      0   \n",
       "2       2.0  2016      6    3  ...         0     0      0    0       0      0   \n",
       "3       3.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "4       4.0  2016      4   12  ...         0     0      0    1       0      0   \n",
       "\n",
       "   wheelchair  wifi  windows  work  \n",
       "0           0     0        0     0  \n",
       "1           0     0        0     0  \n",
       "2           0     0        0     0  \n",
       "3           1     0        0     0  \n",
       "4           0     0        0     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取测试数据\n",
    "dtest = pd.read_csv('./data/RentListingInquries_FE_test.csv')\n",
    "dtest.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#test_id = dtest['listing_id']\n",
    "X_test = dtest\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load 训练好的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存数据\n",
    "xgb_new = pk.load(open(\"xgb_model.pkl\", 'rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 在测试上测试，并生成测试结果提交文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_pred = xgb_new.predict_proba(X_test)\n",
    "\n",
    "out_df1 = pd.DataFrame(y_test_pred)\n",
    "out_df1.columns = [\"high\", \"medium\", \"low\"]\n",
    "\n",
    "out_df = pd.concat([X_test,out_df1], axis = 1)\n",
    "out_df.to_csv(\"xgb_Rent.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
